# coding:utf-8
# 根据mysql中的词典对未标注的文档进行初步标定
import jieba.analyse
import pymysql
import json
import sys
import os


project_path = os.getcwd()
gather_path = os.path.abspath(os.path.dirname(project_path) + os.path.sep + ".")
father_path = os.path.abspath(os.path.dirname(project_path) + os.path.sep + "..")
sys.path.append(gather_path)

from config.config import Config
from bs4 import BeautifulSoup

print ("project path :" + project_path)
print ("project father path:" + father_path)
file_type = '.txt'
conf = Config()
json_dict = {}
data_root_path = father_path + "/abc_project_data/unmark_html/"
data_output_path = father_path + "/abc_project_data/marked_html/"
dictPath = father_path + "/abc_project_data/sql_dict.txt"
sql_mark_dict = father_path + "/abc_project_data/sql_dict.json"
stopword = father_path + "/abc_project_data/stopword.dict"
sql_dict = open(dictPath, 'w')
sql_mark_dict = open(sql_mark_dict,'w')
"""
if os.path.isdir(data_output_path):
    shutil.rmtree(data_output_path)
os.mkdir(data_output_path)
"""
sql_company_name = '''SELECT company_name FROM company_basicinfo group by company_name'''
#sql_person_name = '''SELECT  name FROM people_basicinfo group by name'''
sql_industry_name = '''SELECT  first_indu_name FROM sec_industry_new group by first_indu_name union
    SELECT  second_indu_name FROM sec_industry_new group by second_indu_name union
    SELECT  third_indu_name FROM sec_industry_new group by third_indu_name union
    SELECT  fourth_indu_name FROM sec_industry_new group by fourth_indu_name 
    '''
sql_product_name = '''SELECT com_product FROM kg_com_product group by com_product'''
sql_business_name ='''SELECT com_business FROM kg_com_business group by com_business'''
#sql_event_name = '''SELECT event_name FROM kg_event WHERE length(event_name)<16 group by event_name '''
jieba.analyse.set_stop_words(stopword)


def get_file_name(data_root_path, file_type):
    file_name = []
    for root, dir, files in os.walk(data_root_path):
        for file in files:
            if file_type in file:
                file_name.append(file)
    return file_name


def getMysqlConn(db):
    if db=='test':
        host, port, dbname, user_name, password = conf.get_mysql_test()
    else :
        host, port, dbname, user_name, password = conf.get_mysql_conn()

    conn = pymysql.connect(
        host=host,
        port=int(port),
        user=user_name,
        passwd=password,
        db=dbname,
        charset='utf8',
    )
    return conn

def check_contain_chinese(check_str):
    for ch in check_str:
        if u'\u4e00' <= ch <= u'\u9fff':
            return True
    return False

# 获取开发mysql数据库连接
def extractor_dict(sql_text,db,flag):
    sub_dict = {}
    conn = getMysqlConn(db)
    cur = conn.cursor()
    cur.execute(sql_text)
    word_list = cur.fetchall()
    # print "jieguo count:".count(results)
    for word in word_list:
        if check_contain_chinese(word[0]):
            json_dict[hash(word[0].encode('utf-8'))]=flag
            sql_dict.write(word[0].encode('utf-8') + '\n')
            sub_dict[hash(word[0].encode('utf-8'))] = word[0].encode('utf-8')
    cur.close()
    conn.close()
    return sub_dict


def produce(input_txt, dict,find_dict):
    word_list = list(set(jieba.cut(input_txt, cut_all=False)))
    for word in word_list :
        for key in dict :
            if hash(word.encode('utf-8')) in dict[key].keys():
                find_dict[key].append(word.encode('utf-8'))
    return find_dict


def main():
    dict = {}
    find_dict = {}
    company_dict = extractor_dict(sql_company_name,'test','company')
    #person_dict = extractor_dict(sql_person_name, 'test','person')
    industry_dict = extractor_dict(sql_industry_name,'test', 'industry')
    product_dict = extractor_dict(sql_product_name, 'test','product')
    business_dict = extractor_dict(sql_business_name, 'test','business')
    #event_dict = extractor_dict(sql_event_name, 'online','event')
    json.dump(json_dict,sql_mark_dict,ensure_ascii=False)
    sql_dict.close()
    sql_mark_dict.close()
    print "complete writing sql dict..."

    dict['company'] = company_dict
    #dict['person'] = person_dict
    dict['industry'] = industry_dict
    dict['product'] = product_dict
    dict['business'] = business_dict
    #dict['event'] = event_dict

    """
    jieba.load_userdict(dictPath)

    file_name_list = get_file_name(data_root_path, file_type)
    num = 0

    scan_record = open(data_root_path + 'scan_record.txt', 'w')
    for file in file_name_list:
        num=num+1
        print num
        find_dict['company'] = []
        find_dict['industry'] = []
        find_dict['business'] = []
        find_dict['product'] = []
        find_dict['event'] = []
        find_dict['person'] = []
        scan_record.write(file+'\n')
      
        input_html = open(data_root_path + file, 'r').read().decode('utf-8')
        soup = BeautifulSoup(input_html, 'lxml')
        origin_text = soup.get_text().encode('utf-8')
        find_dict = produce(origin_text, dict,find_dict)
        if find_dict['company'] == []  and find_dict['industry'] == [] and find_dict['person'] == []\
                and find_dict['business'] == [] and find_dict['product'] == [] and find_dict['event'] == []:
            continue
        else:
            output_file = open(data_output_path + file, 'w')
            print "find file :{}".format(file)
            for key in find_dict.keys():
                output_file.write(key + ":" + " ".join(find_dict[key]) + '\n')"""

"""
def main():
    #extract company people and industry in mysql db
    dict={}
    company_list = extractor_dict(sql_company_name)
    person__list = extractor_dict(sql_person_name)
    industry_list = extractor_dict(sql_industry_name)
    dict['company']=company_list
    dict['person'] = person__list
    dict['industry'] = industry_list

    file_name_list = get_file_name(data_root_path, file_type)
    for file in file_name_list:
        input_html = open(data_root_path+file,'r').read().decode('utf-8')
        soup = BeautifulSoup(input_html, 'lxml')
        origin_text = soup.get_text().encode('utf-8')
        find_dict = produce(origin_text,dict)
        if find_dict['company'] ==[] and find_dict['person'] ==[] and find_dict['industry'] ==[]:
            continue
        else:
            output_file = open(data_output_path + file, 'w')
            for key in find_dict.keys():
                output_file.write(key+":"+" ".join(find_dict[key])+'\n')
"""
if __name__ == "__main__":
    main()
